# 导入必要的库
import pandas as pd
import numpy as np
from sklearn.metrics import roc_curve, auc
import matplotlib.pyplot as plt
from sklearn.tree import DecisionTreeClassifier
from sklearn.linear_model import LogisticRegression
import xgboost as xgb
import lightgbm as lgb
from sklearn.preprocessing import StandardScaler

# 读取数据
df1 = pd.read_csv('part-00000-bec7c3f5-6b55-4911-9150-08745df40bae-c000.csv',sep='\\x7f\\x5e',engine='python')
df2 = pd.read_csv('part-00001-bec7c3f5-6b55-4911-9150-08745df40bae-c000.csv',sep='\\x7f\\x5e',engine='python')
df=pd.concat([df1, df2], ignore_index=True)

# 定义要分析的字段列表
score_fields = ['QYZXMODEL', 'HNDGMODEL', 'JXJKMODEL', 'FICOMODEL', 'GRZXMODEL', 'HNGRMODEL', 'GSMODEL']

# 确保 APPLY_DT 的格式正确
df['APPLY_DT'] = pd.to_datetime(df['APPLY_DT'])

# 定义Y标签
df['Y'] = 2  # 默认值为2
df.loc[df['PFLAG_30D'] == 1, 'Y'] = 1  # 逾期30天及以上为1
df.loc[df['PFLAG_30D'] == 0, 'Y'] = 0   # 未逾期为0

# 剔除Y=2的样本
df = df[df['Y'] != 2]

# 准备特征数据
X = df[score_fields].fillna(df[score_fields].mean())
y = df['Y']

# 数据标准化
scaler = StandardScaler()
X_scaled = pd.DataFrame(
    scaler.fit_transform(X),
    columns=score_fields,
    index=X.index  # 保持原始索引
)

# 按日期拆分数据集
cutoff_date = pd.to_datetime('2024-05-31')

# 分离训练验证集和OOT集 - 重新修正这部分代码
train_val_mask = df['APPLY_DT'] <= cutoff_date
X_train_val = X_scaled[train_val_mask]
y_train_val = y[train_val_mask]
X_oot = X_scaled[~train_val_mask]
y_oot = y[~train_val_mask]

# 将训练验证集按6:4比例随机分割
from sklearn.model_selection import train_test_split
X_train, X_val, y_train, y_val = train_test_split(X_train_val, y_train_val, train_size=0.6, random_state=42)

# 定义模型字典
models = {
    'LogisticRegression': LogisticRegression(random_state=42),
    'DecisionTree': DecisionTreeClassifier(random_state=42),
    'XGBoost': xgb.XGBClassifier(random_state=42),
    'LightGBM': lgb.LGBMClassifier(random_state=42)
}

# 存储模型结果
results = {}

# 训练模型并评估
for name, model in models.items():
    print(f'\n训练 {name} 模型...')
    
    # 训练模型
    model.fit(X_train, y_train)
    
    # 针对逻辑回归模型添加详细分析
    if name == 'LogisticRegression':
        # 计算VIF
        from statsmodels.stats.outliers_influence import variance_inflation_factor
        
        # 创建VIF数据框
        vif_data = pd.DataFrame()
        vif_data["变量"] = X_train.columns
        vif_data["VIF"] = [variance_inflation_factor(X_train.values, i) 
                          for i in range(X_train.shape[1])]
        
        # 创建模型系数数据框
        coef_data = pd.DataFrame({
            '变量': X_train.columns,
            '系数': model.coef_[0],
            'Odds比': np.exp(model.coef_[0])
        })
        
        print("\n逻辑回归模型详细结果:")
        print("\n变量VIF值:")
        print(vif_data.to_string(index=False))
        print("\n变量系数和Odds比:")
        print(coef_data.to_string(index=False))
        print(f"\n截距项: {model.intercept_[0]:.4f}")
    
    # 在验证集上评估
    val_proba = model.predict_proba(X_val)[:, 1]
    fpr_val, tpr_val, _ = roc_curve(y_val, val_proba)
    ks_val = max(abs(tpr_val - fpr_val))
    
    # 在OOT集上评估
    oot_proba = model.predict_proba(X_oot)[:, 1]
    fpr_oot, tpr_oot, _ = roc_curve(y_oot, oot_proba)
    ks_oot = max(abs(tpr_oot - fpr_oot))
    
    results[name] = {
        'validation_ks': ks_val,
        'oot_ks': ks_oot
    }
    
    print(f'{name} 验证集 KS: {ks_val:.3f}')
    print(f'{name} OOT集 KS: {ks_oot:.3f}')
    
    # 绘制ROC曲线
    plt.figure(figsize=(12, 5))
    
    # 验证集ROC
    plt.subplot(1, 2, 1)
    plt.plot(fpr_val, tpr_val)
    plt.plot([0, 1], [0, 1], 'k--')
    plt.title(f'{name} - 验证集 ROC (KS={ks_val:.3f})')
    plt.xlabel('False Positive Rate')
    plt.ylabel('True Positive Rate')
    plt.grid(True)
    
    # OOT集ROC
    plt.subplot(1, 2, 2)
    plt.plot(fpr_oot, tpr_oot)
    plt.plot([0, 1], [0, 1], 'k--')
    plt.title(f'{name} - OOT集 ROC (KS={ks_oot:.3f})')
    plt.xlabel('False Positive Rate')
    plt.ylabel('True Positive Rate')
    plt.grid(True)
    
    plt.tight_layout()
    plt.show()

# 找出OOT集上表现最好的模型
best_model = max(results.items(), key=lambda x: x[1]['oot_ks'])
print('\n最终结果:')
print(f'OOT集上表现最好的模型是: {best_model[0]}')
print(f'验证集 KS: {best_model[1]["validation_ks"]:.3f}')
print(f'OOT集 KS: {best_model[1]["oot_ks"]:.3f}')
